import random
from urllib.parse import urlencode
import requests
from pyquery import PyQuery as pq
import pymongo
from multiprocessing.pool import Pool

def get_ip():
    IP = [
        {'http:': 'http://139.196.196.74:80'},
        {'http:': 'http://222.33.192.238:8118'},
        {'http:': 'http://219.150.189.212:9999'},
        {'http:': 'http://125.94.0.250:8080'},
        {'http:': 'http://117.90.2.230:9000'},
        {'http:': 'http://124.42.7.103:80'},
        {'http:': 'http://121.69.37.238:8118'},
        {'http:': 'http://121.69.37.238:8118'},
        {'http:': 'http://117.90.3.218:9000'},
        {'http:': 'http://219.150.189.212:9999'},
        {'http:': 'http://124.42.7.103:80'},
    ]
    IP = random.choice(IP)
    return IP

# 发起第一次请求，获取并返回包含用人单位 url的 results
def get_results(offes, IP):
    params = {
        'start':offes,
        'pageSize': '60',
        'cityId': '489',
        'workExperience': '-1',
        'education': '-1',
        'companyType': '-1',
        'employmentType': '-1',
        'jobWelfareTag':' -1',
        'kw': 'python',
        'kt': '3',
        'lastUrlQuery': {"pageSize": "60", "jl": "489", "kw": "python", "kt": "3"}
    }
    headers = {
        'Cookie': 'adfbid2=0; sts_deviceid=165147cd42b7ff-0159ccc536b33-6114147a-2073600-165147cd42c743; dywem=95841923.y; LastCity=%E9%83%91%E5%B7%9E; LastCity%5Fid=719; ZP_OLD_FLAG=false; _jzqy=1.1533731781.1536626819.4.jzqsr=baidu|jzqct=%E6%99%BA%E8%81%94%E6%8B%9B%E8%81%98.jzqsr=baidu|jzqct=%E6%8B%9B%E8%81%98; adfbid=0; sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22165d33889d6470-076bb4cb7aa928-43480420-2073600-165d33889d8717%22%2C%22%24device_id%22%3A%22165d33889d6470-076bb4cb7aa928-43480420-2073600-165d33889d8717%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E4%BB%98%E8%B4%B9%E5%B9%BF%E5%91%8A%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22http%3A%2F%2Fwww.hao123.com%2Flink%2Fv3%2F%3Fkey%3DpZwYTjCEQLILIz4lpA7ETA-8QhPEUit%253D%26pageid%3Dhao123-pcbdhz-zhaopin%26monkey%3Dm-pcbdhz-zhaopin%26title%3Dzhilian%22%2C%22%24latest_referrer_host%22%3A%22www.hao123.com%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC%22%2C%22%24latest_utm_source%22%3A%22other%22%2C%22%24latest_utm_medium%22%3A%22cnt%22%2C%22%24latest_utm_campaign%22%3A%22121114603%22%7D%7D; dywea=95841923.4583121453675378000.1533647050.1536626818.1536847023.10; dywec=95841923; dywez=95841923.1536847023.10.10.dywecsr=other|dyweccn=121114603|dywecmd=cnt; Hm_lvt_38ba284938d5eddca645bb5e02a02006=1536200992,1536558705,1536626818,1536847023; sts_sg=1; sts_sid=165d3388ad63d2-0ef3b0ee2ecdeb-43480420-2073600-165d3388ad7817; zp_src_url=http%3A%2F%2Fwww.hao123.com%2Flink%2Fv3%2F%3Fkey%3DpZwYTjCEQLILIz4lpA7ETA-8QhPEUit%253D%26pageid%3Dhao123-pcbdhz-zhaopin%26monkey%3Dm-pcbdhz-zhaopin%26title%3Dzhilian; __utma=269921210.927875743.1533647050.1536626819.1536847024.11; __utmc=269921210; __utmz=269921210.1536847024.11.10.utmcsr=other|utmccn=121114603|utmcmd=cnt; _jzqa=1.1848289355499773200.1533647051.1536626819.1536847024.9; _jzqc=1; _jzqx=1.1533647051.1536847024.1.jzqsr=hao123%2Ecom|jzqct=/link/v3/.-; _jzqckmp=1; __xsptplus30=30.11.1536847024.1536847024.1%231%7Cother%7Ccnt%7C121114603%7C%7C%23%23SRVgR3NgzQ-A0tvNrhh0m6t-JAKOL4qL%23; lastchannelurl=https%3A//passport.zhaopin.com/login; qrcodekey=f6e07d45c8e8445498c6eaded2601f2d; firstchannelurl=https%3A//passport.zhaopin.com/login%3Fy7bRbP%3DdpocrKJO3CJO3CJOScnqXh8SlORe5l9auoWtpxhjTCW; JsNewlogin=2144955527; JSloginnamecookie=17737450122; JSShowname=; at=3e4d6c82a3a048ae8bbfa9b915ac98b5; Token=3e4d6c82a3a048ae8bbfa9b915ac98b5; rt=ee9f5b96c7e14848b44b95133d9f07c7; JSsUserInfo=3d692e695671407154775d75506a5c75457759695d695e714c7129772775546a557547775f6959695c71437150775975586a5675427753692f6926714a7154775e755e6a577547775c695f695b71467157775b75296a557546775b69446908711871087752753a6a30754d77596951692a71237158775175446a507550775a695b69517142715d775275286a28754d77586951693e71367158772375256a55754477516952695e71467152775c75516a5f7525773c6957695a714c7136772075546a51754077586959695b714471547752753c6a34753e7755695b695c71407156775e755d6a51754077596958695971377154775f755a6a4a7513770769076950716; uiioit=3d753d6a44640e385a6d5a620f355c6840795b7953390d6b566e203671645575496a42649; loginreleased=1; GUID=28e645d608884118b08b7653389618c9; Hm_lpvt_38ba284938d5eddca645bb5e02a02006=1536848115; urlfrom=121126445; urlfrom2=121126445; adfcid=none; adfcid2=none; ZL_REPORT_GLOBAL={%22//i%22:{%22actionid%22:%229de0bed2-a4e2-4ea4-8c73-5973061664d2-i%22%2C%22funczone%22:%22back_best_for_you%22}%2C%22//jobs%22:{%22actionid%22:%22b17d0db2-82e1-46a2-8bf0-f8b039bf88ed-jobs%22%2C%22funczone%22:%22dtl_best_for_you%22}%2C%22sou%22:{%22actionid%22:%228df75f3b-5900-44ca-a31b-f087c659440b-sou%22%2C%22funczone%22:%22smart_matching%22}}; dyweb=95841923.13.9.1536847967277; __utmb=269921210.11.9.1536847967291; sts_evtseq=28',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
        'Host': 'fe-api.zhaopin.com',
        'Referer': 'https://sou.zhaopin.com/?pageSize=60&jl=489&kw=python&kt=3',
        'Origin': 'https://sou.zhaopin.com',
    }
    # 此处用 urlencode()方法构造请求的GET参数
    url = 'https://fe-api.zhaopin.com/c/i/sou?' + urlencode(params)
    try:
        rep = requests.get(url, headers=headers, proxies=IP, timeout=5)
        if rep.status_code == 200:
            code = rep.json()  # 将返回结果转换为json格式
            if code.get('data'):  # 解析获得的json字典
                data = code.get('data')
                if data.get('results'):
                    results = data.get('results')
                    return results
                print('提取 results 失败')
                return None
            print('提取 data 失败')
            return None
        return None
    except requests.ConnectionError:
        print('请求索引页出错')
        return None

# 此处遍历获取用人单位主页 url
def get_urls(results):
    for item in results:
        url = item.get('positionURL') 
        yield url

def get_data(url, IP):
    headers = {
        'cookie': 'adfbid2=0; sts_deviceid=165147cd42b7ff-0159ccc536b33-6114147a-2073600-165147cd42c743; dywem=95841923.y; LastCity=%E9%83%91%E5%B7%9E; LastCity%5Fid=719; ZP_OLD_FLAG=false; _jzqy=1.1533731781.1536626819.4.jzqsr=baidu|jzqct=%E6%99%BA%E8%81%94%E6%8B%9B%E8%81%98.jzqsr=baidu|jzqct=%E6%8B%9B%E8%81%98; adfbid=0; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22165d33889d6470-076bb4cb7aa928-43480420-2073600-165d33889d8717%22%2C%22%24device_id%22%3A%22165d33889d6470-076bb4cb7aa928-43480420-2073600-165d33889d8717%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E4%BB%98%E8%B4%B9%E5%B9%BF%E5%91%8A%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22http%3A%2F%2Fwww.hao123.com%2Flink%2Fv3%2F%3Fkey%3DpZwYTjCEQLILIz4lpA7ETA-8QhPEUit%253D%26pageid%3Dhao123-pcbdhz-zhaopin%26monkey%3Dm-pcbdhz-zhaopin%26title%3Dzhilian%22%2C%22%24latest_referrer_host%22%3A%22www.hao123.com%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC%22%2C%22%24latest_utm_source%22%3A%22other%22%2C%22%24latest_utm_medium%22%3A%22cnt%22%2C%22%24latest_utm_campaign%22%3A%22121114603%22%7D%7D; dywec=95841923; Hm_lvt_38ba284938d5eddca645bb5e02a02006=1536200992,1536558705,1536626818,1536847023; sts_sg=1; zp_src_url=http%3A%2F%2Fwww.hao123.com%2Flink%2Fv3%2F%3Fkey%3DpZwYTjCEQLILIz4lpA7ETA-8QhPEUit%253D%26pageid%3Dhao123-pcbdhz-zhaopin%26monkey%3Dm-pcbdhz-zhaopin%26title%3Dzhilian; __utmc=269921210; _jzqa=1.1848289355499773200.1533647051.1536626819.1536847024.9; _jzqc=1; _jzqx=1.1533647051.1536847024.1.jzqsr=hao123%2Ecom|jzqct=/link/v3/.-; _jzqckmp=1; __xsptplus30=30.11.1536847024.1536847024.1%231%7Cother%7Ccnt%7C121114603%7C%7C%23%23SRVgR3NgzQ-A0tvNrhh0m6t-JAKOL4qL%23; lastchannelurl=https%3A//passport.zhaopin.com/login; qrcodekey=f6e07d45c8e8445498c6eaded2601f2d; firstchannelurl=https%3A//passport.zhaopin.com/login%3Fy7bRbP%3DdpocrKJO3CJO3CJOScnqXh8SlORe5l9auoWtpxhjTCW; JsNewlogin=2144955527; JSloginnamecookie=17737450122; JSShowname=; at=3e4d6c82a3a048ae8bbfa9b915ac98b5; Token=3e4d6c82a3a048ae8bbfa9b915ac98b5; rt=ee9f5b96c7e14848b44b95133d9f07c7; JSsUserInfo=3d692e695671407154775d75506a5c75457759695d695e714c7129772775546a557547775f6959695c71437150775975586a5675427753692f6926714a7154775e755e6a577547775c695f695b71467157775b75296a557546775b69446908711871087752753a6a30754d77596951692a71237158775175446a507550775a695b69517142715d775275286a28754d77586951693e71367158772375256a55754477516952695e71467152775c75516a5f7525773c6957695a714c7136772075546a51754077586959695b714471547752753c6a34753e7755695b695c71407156775e755d6a51754077596958695971377154775f755a6a4a7513770769076950716; uiioit=3d753d6a44640e385a6d5a620f355c6840795b7953390d6b566e203671645575496a42649; loginreleased=1; urlfrom=121126445; urlfrom2=121126445; adfcid=none; adfcid2=none; GUID=f91993e6487243279490fc35c66ec09a; Hm_lpvt_38ba284938d5eddca645bb5e02a02006=1536891945; dywez=95841923.1536893395.12.11.dywecsr=sou.zhaopin.com|dyweccn=(referral)|dywecmd=referral|dywectr=undefined|dywecct=/; __utmz=269921210.1536893395.13.11.utmcsr=sou.zhaopin.com|utmccn=(referral)|utmcmd=referral|utmcct=/; Hm_lvt_80e552e101e24fe607597e5f45c8d2a2=1536560688,1536847969,1536893395,1536897516; campusOperateJobUserInfo=7fbb2ef9-93d9-411d-ad0e-77feecba8a47; zg_did=%7B%22did%22%3A%20%22165d67621d33cb-00444211d6a037-43480420-1fa400-165d67621d4459%22%7D; zg_08c5bcee6e9a4c0594a5d34b79b9622a=%7B%22sid%22%3A%201536901390808%2C%22updated%22%3A%201536901390906%2C%22info%22%3A%201536901390810%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22%22%7D; ZL_REPORT_GLOBAL={%22//i%22:{%22actionid%22:%229de0bed2-a4e2-4ea4-8c73-5973061664d2-i%22%2C%22funczone%22:%22back_best_for_you%22}%2C%22//jobs%22:{%22actionid%22:%22f2ecae37-9853-48ce-bc10-227568281b22-jobs%22%2C%22funczone%22:%22dtl_best_for_you%22}%2C%22sou%22:{%22actionid%22:%220abe7481-cd63-4825-9707-af7fa77bbfca-sou%22%2C%22funczone%22:%22smart_matching%22}%2C%22company%22:{%22actionid%22:%22abcaa21f-209f-4d6a-b097-af2a98c82634-company%22%2C%22funczone%22:%22hiring_jd%22}}; dywea=95841923.4583121453675378000.1533647050.1536896312.1536902002.14; dyweb=95841923.2.10.1536902002; sts_sid=165d67f7509413-09b134c334a529-43480420-2073600-165d67f750a357; sts_evtseq=2; __utma=269921210.927875743.1533647050.1536896312.1536902002.15; __utmt=1; __utmb=269921210.2.10.1536902002; Hm_lpvt_80e552e101e24fe607597e5f45c8d2a2=1536902002; referrerUrl=; stayTimeCookie=1536902002825',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
    }
    try:
        rep = requests.get(url, headers=headers, proxies=IP, timeout=5)
        if rep.status_code == 200:
            data = rep.text
            # 提取数据
            doc = pq(data)
            titles = doc('.new-info > ul')
            title = titles('li > .info-h3').text()       # 标题
            if not title:
                return None
            pay = titles('div > strong').text()          # 薪资
            company = titles('.clearfix .company a').text()
            workexperience = titles('li > .info-three > span').text()
            zhize = doc('.pos-info > .pos-info-in > .responsibility p').text()
            fuli = doc('.pos-info > .pos-info-in > .welfare > ul > li').text()
            print(title)
            yield {
                    'title':title, 
                    'pay': pay, 
                    'company': company, 
                    'workexperience': workexperience, 
                    'zhize': zhize, 
                    'fuli': fuli 
                    }
        return None
    except requests.ConnectionError as e:
        print('二次请求索引页出错', e)
        return None    

# 此方法用于将数据插入数据库
def save_mongo(rep):
    # 连接数据库
    client = pymongo.MongoClient(host='localhost', port=27017)
    # 指定数据库
    db = client.test
    # 指定集合（表）
    collection = db.zhilian
    zhilian = {
        'title': rep['title'], 
        'pay': rep['pay'], 
        'company': rep['company'], 
        'workexperience': rep['workexperience'], 
        'zhize': rep['zhize'], 
        'fuli': rep['fuli'] 
        }
    collection.insert_one(zhilian)
#    print('插入成功', zhilian)

def main(offes):
    IP = get_ip()
    results = get_results(offes, IP)
    # 此处需要多次调用 获得url
    for url in get_urls(results):
        for rep in get_data(url, IP):
            print('*********************')
            save_mongo(rep)
            print('============')

if __name__ == '__main__':
#    main()
    # 创建进程池
    pool = Pool(4)
    offes = ([str(x * 60) for x in range(1, 40)])
    print(offes)
    pool.map(main, offes)
    pool.close()
    pool.join()